package cnki.fetch;

import cnki.db.sql.Operator;
import cnki.entity.ArticleAbstractInfoEntity;
import cnki.entity.PeriodicalInfoEntity;
import cnki.proxy.ProxyImpl;
import cnki.util.FileManager;
import cnki.util.ListTool;
import com.han.http.client.exception.NullContentMatchedException;
import com.han.http.util.Regex;
import org.json.JSONObject;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.net.Proxy;
import java.util.*;
import java.util.concurrent.Executors;

import static com.han.http.util.Print.printlnErr;
import static com.han.http.util.Print.printlnInfo;

/**
 * @author: Han
 * @date: 2018/9/29 16:12
 * @dscription: 抓取期刊文章相关
 */
public class ArticleFetcher {
    private Operator operator;

    private ArticleFetcher() {
        operator = Operator.getInstance();
    }

    public static void main(String... args) {
        var fetcher = new ArticleFetcher();
        fetcher.getContent();
    }

    private void getArticleUrl() {
        var periodicalInfoEntityList = operator.queryPeriodicalInfo();
        var splitList = new ListTool<PeriodicalInfoEntity>().splitListByCount(periodicalInfoEntityList, 10);
        var executor = Executors.newFixedThreadPool(splitList.size());
        for (var i = 0; i < splitList.size(); i++) {
            var tempList = splitList.get(i);
            ArticleUrlThread thread = new ArticleUrlThread(tempList);
            thread.setName("Thread-文章链接-" + i);
            executor.submit(thread);
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        executor.shutdown();
    }

    private void getContent() {
        var count = 87;
        var executors = Executors.newFixedThreadPool(10);
        for (var j = 1; j <= count; j++) {
            var start = (j - 1) * 100000 + 1;
            var articleAbstractInfoEntityList = operator.queryAbstractInfo(start);
            var splitList = new ListTool<ArticleAbstractInfoEntity>().splitListByCount(articleAbstractInfoEntityList, 10);
            for (var i = 0; i < splitList.size(); i++) {
                var tempAbstractInfoList = splitList.get(i);
                var thread = new ArticleContentThread(tempAbstractInfoList);
                thread.setName("Thread-文章概要-" + i);
                executors.submit(thread);
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        }
        executors.shutdown();
    }
}

/**
 * 抓取摘要等信息
 */
class ArticleContentThread extends Thread {
    private FileManager fileManager;
    private ProxyImpl proxyImpl;
    private Proxy httpProxy;
    private Operator operator;
    private List<ArticleAbstractInfoEntity> articleAbstractInfoEntityList;

    ArticleContentThread(List<ArticleAbstractInfoEntity> articleAbstractInfoEntityList) {
        this.articleAbstractInfoEntityList = articleAbstractInfoEntityList;
        operator = Operator.getInstance();
        proxyImpl = new ProxyImpl();
        httpProxy = proxyImpl.getHttpProxy();
        fileManager = new FileManager(ArticleContentThread.class);
    }

    @Override
    public void run() {
        getArticles();
    }

    /**
     * 遍历没有获取信息的文章url,填充内容
     */
    private void getArticles() {
        var random = new Random();
        var repeatCount = 0;
        for (int i = 0; i < articleAbstractInfoEntityList.size(); i++) {
            var entity = articleAbstractInfoEntityList.get(i);
            //提示信息
            var infoMsg = entity.getPeriodicalId() + " " + entity.getId() + "," + entity.getUrl();
            try {
                //使用链接填充细节
                fillDetail(entity);
                //重复数置零
                repeatCount = 0;
                //写入日志
                //fileManager.write(infoMsg + "抓取成功", true);
                printlnInfo(getName() + " " + infoMsg + "抓取成功");
            } catch (IOException e) {
                printlnErr(getName() + " " + httpProxy + "不可用,换");
                httpProxy = proxyImpl.getHttpProxy();
                //同一个链接最多重复十次
                if (repeatCount++ < 10) {
                    i--;
                } else {
                    fileManager.write(infoMsg + "抓取失败", true);
                    printlnErr(getName() + " " + infoMsg + "抓取失败");
                }
            } catch (Exception e) {
                fileManager.write(infoMsg + "抓取失败", true);
                printlnErr(getName() + " " + entity.getPeriodicalId() + " " + entity.getId() + "," + entity.getUrl() +
                        "抓取失败");
                e.printStackTrace();
            } finally {
                //抓取之前先暂停,避免和
                int interval = random.nextInt(10) + 10;
                try {
                    Thread.sleep(interval);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 填充内容
     *
     * @param entity 文章实体
     */
    private void fillDetail(ArticleAbstractInfoEntity entity) throws IOException {
        //请求头
        var headers = new HashMap<String, String>(5);
        headers.put("Cache-Control", "max-age=0");
        headers.put("Upgrade-Insecure-Requests", "1");
        headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3528.4 Safari/537.36");
        headers.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
        headers.put("Accept-Encoding", "gzip, deflate");
        headers.put("Accept-Language", "en,zh-CN;q=0.9,zh;q=0.8");

        //链接页面
        var url = entity.getUrl();
        var connection = Jsoup.connect(url);
        connection.headers(headers);
        connection.timeout(1000 * 10);
        connection.proxy(httpProxy);
        //解析页面
        var document = connection.get();
        //标题
        var titleEle = document.getElementsByClass("wxTitle").first();
        var titleEles = titleEle.getElementsByClass("title");
        var title = titleEles.text();
        entity.setTitle(title);
        //作者
        var authorEle = titleEle.getElementsByClass("author");
        var authorEles = authorEle.select("span");
        var authorSB = new StringBuilder();
        //遍历作者信息
        var random = new Random();
        var repeatCount = 0;
        for (var i = 0; i < authorEles.size(); i++) {
            var element = authorEles.get(i);
            int interval = random.nextInt(10) + 10;
            try {
                Thread.sleep(interval);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            try {
                var aEle = element.getElementsByTag("a").first();
                var authorToken = aEle.attr("onclick");
                var author = getAuthor(authorToken, url);
                authorSB.append(author).append("\n");
                //重置重复次数
                repeatCount = 0;
            } catch (IOException e) {
                printlnErr(getName() + " " + httpProxy + "不可用,换");
                httpProxy = proxyImpl.getHttpProxy();
                //同一个链接最多重复10次
                if (repeatCount++ < 10) {
                    i--;
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        entity.setAuthor(authorSB.toString());
        //解析基本信息
        var wxBaseInfoEle = document.getElementsByClass("wxBaseinfo").first();
        var pEles = wxBaseInfoEle.getElementsByTag("p");
        //遍历所有基本信息
        for (var element : pEles) {
            var labelEles = element.getElementsByTag("label");
            var label = labelEles.text();
            if ("摘要：".equals(label)) {
                //解析摘要
                var digestEle = element.getElementById("ChDivSummary");
                var digest = digestEle.text();
                entity.setDigest(digest);
            } else if ("基金：".equals(label)) {
                //解析基金
                var fundEle = element.getElementsByTag("a");
                var fundSB = new StringBuilder();
                for (var tempEle : fundEle) {
                    var fundName = tempEle.text();
                    fundSB.append(fundName);
                }
                entity.setFund(fundSB.toString());
            } else if ("关键词：".equals(label)) {
                //解析关键词
                var keywordEle = element.getElementsByTag("a");
                var keywordSB = new StringBuilder();
                for (var tempEle : keywordEle) {
                    var keywordName = tempEle.text();
                    keywordSB.append(keywordName);
                }
                entity.setKeyword(keywordSB.toString());
            } else if ("DOI：".equals(label)) {
                //解析DOI
                var doi = element.text().replace("DOI：", "");
                entity.setDoi(doi);
            } else if ("分类号：".equals(label)) {
                //解析分类号
                var typeNum = element.text().replace("分类号：", "");
                entity.setTypeNum(typeNum);
            }
        }
        var downloadEle = wxBaseInfoEle.getElementById("DownLoadParts");
        var cajDownEle = downloadEle.getElementById("cajDown");
        var cajUrl = cajDownEle.attr("href");
        var pdfDownEle = downloadEle.getElementById("pdfDown");
        var pdfUrl = pdfDownEle.attr("href");
        entity.setCajUrl(cajUrl);
        entity.setPdfUrl(pdfUrl);
        //更新文章信息
        operator.updateAbstractInfo(entity);
    }

    private String getAuthor(String token, String oriUrl) throws IOException {
        var authorSB = new StringBuilder();
        //解析出作者页面的url
        String name;
        String url;
        try {
            var a = Regex.matchString(token, "(?<=\\(')((?!').)*(?=',')");
            name = Regex.matchString(token, "(?<=',')((?!').)*(?=',')");
            var c = Regex.matchString(token, "(?<=',')((?!').)*(?='\\))");
            url = String.format("http://kns.cnki.net/kcms/detail/knetsearch.aspx?sfield=%s&skey=%s&code=%s",
                    a, name, c);
        } catch (NullContentMatchedException ignore) {
            authorSB.append("姓名:").append("\t");
            authorSB.append("单位:").append("\t");
            authorSB.append("方向:");
            return authorSB.toString();
        }
        //设置请求头
        var headers = new HashMap<String, String>(9);
        headers.put("Host", "kns.cnki.net");
        headers.put("Connection", "keep-alive");
        headers.put("Cache-Control", "max-age=0");
        headers.put("Upgrade-Insecure-Requests", "1");
        headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3534.4 Safari/537.36");
        headers.put("Refer", oriUrl);
        headers.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
        headers.put("Accept-Encoding", "gzip, deflate");
        headers.put("Accept-Language", "en,zh-CN;q=0.9,zh;q=0.8");
        headers.put("Cookie", "ecp_uid5=bdbef3f33879546a274b164318c808c9; ASP.NET_SessionId=kdh0pl0kwyxywsccekzw1zwz; Ecp_ClientId=7180905112301047220; LID=WEEvREcwSlJHSldTTEYzVDhsOWprSUl1UFlPNUI1MmlaWFpEUVNMUTNnVT0=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4IQMovwHtwkF4VYPoHbKxJw!!; SID_kcms=124106; SID_krsnew=125132; UM_distinctid=165a7c1e6c92d3-043eff1ec4e2a-6154702a-1fa400-165a7c1e6ca2fc; CNZZDATA3258975=cnzz_eid%3D639839087-1536113814-http%253A%252F%252Fnavi.cnki.net%252F%26ntime%3D1536113814; cnkiUserKey=6fa3c02d-fb8a-f88e-4838-48c6675ce0af; SID_kns=123107; Ecp_IpLoginFail=18090560.174.195.245; SID_klogin=125142; _pk_ref=%5B%22%22%2C%22%22%2C1536117828%2C%22http%3A%2F%2Fnavi.cnki.net%2Fknavi%2FJournalDetail%3Fpcode%3DCJFD%26pykm%3DGGGL%2");
        //获取解析对象
        var connection = Jsoup.connect(url);
        connection.headers(headers);
        connection.timeout(1000 * 30);
        connection.proxy(httpProxy);
        //解析页面
        var document = connection.get();
        var baseInfoEle = document.getElementsByClass("baseInfo").first();
        var orgn = "";
        var doma = "";
        try {
            var nameEle = baseInfoEle.getElementsByClass("name").first();
            name = nameEle.text();
            var orgnEle = baseInfoEle.getElementsByClass("orgn").first();
            orgn = orgnEle.text();
            var domaEle = baseInfoEle.getElementsByClass("doma").first();
            doma = domaEle.text();
        } catch (NullPointerException ignore) {
        }
        authorSB.append("姓名:").append(name).append("\t");
        authorSB.append("单位:").append(orgn).append("\t");
        authorSB.append("方向:").append(doma);

        return authorSB.toString();
    }
}

/**
 * 根据期刊信息,抓取指定刊号文章链接
 */
class ArticleUrlThread extends Thread {
    private FileManager fileManager;
    private List<PeriodicalInfoEntity> periodicalInfoEntityList;
    private Operator operator;
    private Proxy httpProxy;
    private ProxyImpl proxyImpl;

    ArticleUrlThread(List<PeriodicalInfoEntity> periodicalInfoEntityList) {
        this.periodicalInfoEntityList = periodicalInfoEntityList;
        operator = Operator.getInstance();
        proxyImpl = new ProxyImpl();
        httpProxy = proxyImpl.getHttpProxy();
        fileManager = new FileManager(ArticleUrlThread.class);
    }

    @Override
    public void run() {
        for (var periodicalEntity : periodicalInfoEntityList) {
            getArticleUrl(periodicalEntity);
            printlnInfo(getName() + " ID:" + periodicalEntity.getId() + "抓取完毕");
        }
    }

    private void getArticleUrl(PeriodicalInfoEntity periodicalInfoEntity) {
        //请求头
        var headers = new HashMap<String, String>(5);
        headers.put("Host", "navi.cnki.net");
        headers.put("Connection", "keep-alive");
        headers.put("Content-Length", "0");
        headers.put("Accept", "*/*");
        headers.put("Origin", "http://navi.cnki.net");
        headers.put("X-Requested-With", "XMLHttpRequest");
        headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3534.4 Safari/537.36");
        headers.put("Referer", "http://navi.cnki.net/knavi/JournalDetail?pcode=CJFD&pykm=JSJX");
        headers.put("Accept-Encoding", "gzip, deflate");
        headers.put("Accept-Language", "en,zh-CN;q=0.9,zh;q=0.8");

        //拼接链接
        var baseid = periodicalInfoEntity.getBaseid();
        var rowUrl = "http://navi.cnki.net/knavi/JournalDetail/GetArticleList?year=%s&issue=%s&pykm=" + baseid
                + "&pageIdx=0&pcode=CJFD";
        //刊号
        var issue = periodicalInfoEntity.getIssue();
        //获取目标刊号
        var issueList = getDownloadIssue(issue);

        //期刊id
        var id = periodicalInfoEntity.getId();

        //成功抓取的刊号
        var succeedIssueList = new ArrayList<String[]>();
        //当前重复次数
        var repeatCount = 0;
        var random = new Random();
        for (var i = 0; i < issueList.size(); i++) {
            //取出刊号
            var issStrArr = issueList.get(i);
            var dt = issStrArr[0];
            var dd = issStrArr[1];
            //拼接链接
            var url = String.format(rowUrl, dt, dd.replace("No.", ""));
            //设置链接对象
            Connection connection = Jsoup.connect(url);
            connection.timeout(1000 * 10);
            connection.headers(headers);
            connection.proxy(httpProxy);
            //拼接信息链接
            var infoMsg = id + "\t" + periodicalInfoEntity.getName() + "\t" + dt + "-" + dd;
            try {
                //以post方式获取页面数据
                Document document = connection.post();
                parseBody(document, id, dt, dd);
                //加入成功抓取的刊号
                succeedIssueList.add(issStrArr);
                //重复值置零
                repeatCount = 0;
                //写入成功
                //fileManager.write(infoMsg + "抓取成功", true);
                printlnInfo(getName() + " " + infoMsg + " 获取成功");
            } catch (IOException e) {
                //如果抓取不成功,换代理重试
                printlnErr(getName() + " " + httpProxy + "不可用,换");
                httpProxy = proxyImpl.getHttpProxy();
                //同一个期刊名最多重复十次
                if (repeatCount++ < 10) {
                    i--;
                } else {
                    fileManager.write(infoMsg + "抓取失败", true);
                    printlnErr(getName() + " " + infoMsg + " 获取失败");
                }
            } finally {
                try {
                    //最多休眠200毫秒,最少休眠100毫秒
                    var interval = random.nextInt(10) + 10;
                    Thread.sleep(interval);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        }
        //修改成功刊号
        for (var iss : succeedIssueList) {
            issue = modifyIssueFetched(issue, iss);
            periodicalInfoEntity.setIssue(issue);
        }
        //更新刊号
        operator.updatePeriodicalInfoIssue(periodicalInfoEntity);
    }

    /**
     * 修改目标刊号为已被抓取状态
     *
     * @param rowIssue 总刊号
     * @param iss      已被抓取刊号
     * @return 修改后的总刊号
     */
    private String modifyIssueFetched(String rowIssue, String[] iss) {
        var rowObj = new JSONObject(rowIssue);
        var dtArr = rowObj.getJSONArray(iss[0]);
        for (var i = 0; i < dtArr.length(); i++) {
            var dtObj = dtArr.getJSONObject(i);
            var dtStr = dtObj.getString("dd");
            if (dtStr.equals(iss[1])) {
                dtObj.put("fetched", true);
            }
        }
        return rowObj.toString();
    }

    /**
     * 解析内容
     *
     * @param document 文本内容
     * @param dt       dt
     * @param dd       dd
     */
    private void parseBody(Document document, int id, String dt, String dd) {
        Elements catalogEle = document.getElementsByClass("name");
        //遍历元素集,获取里面的链接
        for (Element titleEle : catalogEle) {
            try {
                Element tempEle = titleEle.select("a").first();
                if (Objects.isNull(tempEle)) {
                    continue;
                }
                String href = tempEle.attr("href");
                String dbCode = "";
                try {
                    dbCode = Regex.matchString(href, "(?<=dbCode=)((?!&).)*");
                } catch (NullContentMatchedException ignore) {
                }
                String fileName = "";
                try {
                    fileName = Regex.matchString(href, "(?<=filename=)((?!&).)*");
                } catch (NullContentMatchedException ignore) {
                }
                String dbName = "";
                try {
                    dbName = Regex.matchString(href, "(?<=tableName=)((?!&).)*");
                } catch (NullContentMatchedException ignore) {
                }
                String articleUrl = String.format("http://kns.cnki.net/kcms/detail/detail" +
                        ".aspx?dbcode=%s&filename=%s&dbname=%s", dbCode, fileName, dbName);
                ArticleAbstractInfoEntity entity = new ArticleAbstractInfoEntity(articleUrl, id + "", "", "", "", "", "",
                        "", "", dt + "-" + dd, "", "");
                operator.insertAbstractInfo(entity);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    /**
     * 根据需要抓取的年份,抽出未被抓取刊号,如果dts为空,查询所有未被抓取的刊号
     *
     * @param issue 总刊号
     * @param dts   需要抓取的年份
     * @return 符合要求的刊号
     */
    private List<String[]> getDownloadIssue(String issue, String... dts) {
        var rowObj = new JSONObject(issue);
        var issueList = new ArrayList<String[]>();
        if (dts.length == 0) {
            var keySet = rowObj.keySet();
            dts = new String[keySet.size()];
            var i = 0;
            for (var key : keySet) {
                dts[i++] = key;
            }
        }
        //遍历年份
        for (var dtStr : dts) {
            var ddArray = rowObj.getJSONArray(dtStr);
            if (ddArray.length() == 0) {
                printlnInfo("没有这个dt:" + dtStr);
            }
            //遍历二级刊号
            for (var i = 0; i < ddArray.length(); i++) {
                var ddObj = ddArray.getJSONObject(i);
                //只放入未被抓取的刊号
                var isFetched = ddObj.getBoolean("fetched");
                if (!isFetched) {
                    var ddStr = ddObj.getString("dd");
                    String[] iss = {dtStr, ddStr};
                    issueList.add(iss);
                }
            }
        }
        return issueList;
    }
}
